# -*- coding: utf-8 -*-
'''
Created on 22-10-2012

@author: LONG HOANG GIANG
'''

import sys
import os
from urlparse import urljoin
sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
from CrawlerLib import commonlib, html2text
from Model.storymodelv2 import StoryModel
import datetime
import re
import workerpool


class MotSach():
    
    includeTitle = False
    
    def __init__(self, url):
        self.url = url
        
    def getListChapter(self):
        tree = commonlib.loadweb(self.url).build_tree()
        data = []
        for item in tree.xpath("//div[@class='right_menu_item' and contains(@id, 'c')]/a"):
            title = commonlib.normalize_str(item.text)
            title = title.strip('-')
            link = item.get('href')
            link = urljoin(self.url, link)
            data.append({'title': title, 'url': link})
        return data
    
    def getDetail(self, url, title=''):
        print '> getDetail({0})'.format(url)
        tree = commonlib.loadweb(url).build_tree()
        contentNode = tree.xpath("//div[@id='motsach_content_body']")
        commonlib.Etree.clean_elements(contentNode, 'div', {'class': 'story_author'})
        commonlib.Etree.clean_elements(contentNode, 'script')
        commonlib.Etree.clean_elements(contentNode, 'input')
        commonlib.Etree.clean_elements(contentNode, 'div', {'class': 'navigator_bottom'})
        commonlib.Etree.clean_elements(contentNode, 'div', {'id': 'chapter_navigator_box_top'})
        commonlib.Etree.clean_elements(contentNode, 'div', {'class': 'menu_icon'})
        commonlib.Etree.clean_elements(contentNode, 'div', {'class': 'story_title'})
        commonlib.Etree.clean_elements(contentNode, 'div', {'id': 'menu_icon_box'})
        commonlib.Etree.clean_elements(contentNode, 'div', {'class': 'story_chapter'})
        html = commonlib.Etree.tostring(contentNode)
        if re.search(r'''<div id="dropcap" class="dropcap_(.+)"><span class="hidden">(.+)</span></div>''', html):
            html = re.sub(r'''<div id="dropcap" class="dropcap_(.+)"><span class="hidden">(.*)</span></div>''', "\\2", html)
        else:
            html = re.sub(r'''<div id="dropcap" class="dropcap_(.+)"><span class="hidden">(.*)</span></div>''', "\\1", html)
        html = re.sub(r'''<div class="story_chapter">([^<]+)</div>''', "<b>\\1</b>", html)
        
        content = html2text.html2text(html).strip()
        
        content = re.sub(r"\n", "<br />", content)
        content = re.sub(ur"<b> *Chương \d+ *-*</b> *<br[^>]*> *<br[^>]*>", "", content)
        if title != '':
            content = '''<b>{0}</b><br /><hr /><br />{1}'''.format(title, content)
        
        print 'DETAIL: {0}'.format(content[:200])
        return content
    
    def loadHtml2Cache(self, item):
        commonlib.loadweb(item['url'])
    
    
    def process(self):
        smodel = StoryModel(DATABASE)
        smodel.open()
        data = self.getListChapter()
        
        pool = workerpool.WorkerPool(size=10)
        pool.map(self.loadHtml2Cache, data)
        pool.shutdown()
        pool.wait()
        
        for chapter in data:
            if self.includeTitle:
                content = self.getDetail(chapter['url'], chapter['title'])
            else:
                content = self.getDetail(chapter['url'])
            smodel.add_story(commonlib.toUpper(chapter['title']), content, catid=0)
        smodel.close()
    
    
if __name__ == '__main__':
    
    DATABASE = '/longhoanggiang/motsach/database/{0}'.format('qgldvs')
    
    ms = MotSach('http://motsach.info/story.php?story=quang_ganh_lo_di_vui_song&chapter=000')
    ms.includeTitle = True
    ms.process()
    
    
    print '> Finished at {0}'.format(datetime.datetime.now())
    os._exit(1)
